package resourcedownloader;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.text.DecimalFormat;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Main {

	public static void main(String[] args) {
		try {
			int first = 1; // the first first level URL to be explored, initially 1
			String folderUrl = ""; // folder to save resources
			String startUrl = ""; // the start URL
			if ( args.length > 0 ) {
				first = Integer.parseInt( args[0] ); // to continue from a checkpoint pass number as argument
				folderUrl = args[1];
				startUrl = args[2];
			}
			Main.extractFirstLevel( first, folderUrl, startUrl);
		}
		catch (Exception e) {
			e.printStackTrace();
		}
	}

	private static void extractFirstLevel( int first, String urlFolder, String startUrl ) throws IOException {
		URL url = new URL(startUrl); // the start URL
		System.out.println( "--> Tentando: " + startUrl);
		InputStream is = url.openStream();
		System.out.println( "--> URL inicial CONFIRMADO");
		System.out.println( "--> Iniciando coleta a partir do recurso de primeiro nivel: " + first );
		
		BufferedReader br = new BufferedReader( new InputStreamReader( is ) );
		// the pattern used to find first level URLs
		// example: http://www.foo.bar/section1, http://www.foo.bar/section2, http://www.foo.bar/section3
		Pattern pat = Pattern.compile("http://www\\.foo\\.bar/section[0-9]");
		
		String line = "";
		DecimalFormat df = new DecimalFormat("000");
		int folder = 0;
		Set<String> visited = new HashSet<String>();
		while ( (line = br.readLine()) != null ) {
			Matcher mat = pat.matcher( line );
			
			while ( mat.find() ) {
				if (folder < (first-1)) {
					folder++;
					continue; // skip first level URL
				}

				System.out.print("----> " + df.format( ++folder ));
				String foundUrl = mat.group();
				if (!visited.add(foundUrl)) {
					System.out.println(" - Ignorando por duplicacao: " + foundUrl);
					continue;
				}
				System.out.println(" - Tentando: " + foundUrl);
				Main.extractSecondLevel(foundUrl, new File( "data/" + urlFolder + "/" + df.format( folder )));
			}
		}

		br.close();
	}
	
	private static void extractSecondLevel( String urlString, File folder ) {
		if ( !folder.exists() ) {
			folder.mkdir();
		}
		
		try {
			URL url = new URL(urlString);
			InputStream is = url.openStream();
			System.out.println("----> URL de primeiro nivel CONFIRMADO");
			
			BufferedReader br = new BufferedReader( new InputStreamReader( is ) );
			// the pattern used to find second level (resource level) URLs
			// example: http://www.foo.bar/section1/image01.jpg, http://www.foo.bar/section1/image02.jpg
			Pattern pat = Pattern.compile("http://www\\.foo\\.bar/section[0-9]/image[0-9]+\\.jpg");
			
			String line = "";
			DecimalFormat df = new DecimalFormat("00");
			int file = 0;
			while ( (line = br.readLine()) != null ) {
				Matcher mat = pat.matcher( line );
				
				while ( mat.find() ) {
					System.out.print("------> " + df.format( ++file ));
					String foundUrl = mat.group();
					System.out.println(" - Tentando: " + foundUrl);
					Main.loadResource(foundUrl, new File( folder, df.format(file) + ".jpg") );
				}
			}
			
			br.close();
		}
		catch (Exception e) {
			System.out.println("----> URL de primeiro nivel FALHOU");
		}
	}

	private static void loadResource(String urlString, File arquivo) {
		try {
			URL url = new URL( urlString );
			InputStream is = url.openStream();
			System.out.println("------> URL de segundo nivel CONFIRMADO");
			
			FileOutputStream fos = new FileOutputStream( arquivo );
			
			int b;
			while ( (b = is.read()) != -1 ) {
				fos.write(b);
			}
			
			fos.close();
			fos.flush();
			is.close();
		} catch (Exception e) {
			System.out.println("------> URL de segundo nivel FALHOU");
		}
	}
}
